import scrapy
from poetry.items import AuthorItem
import time

class Gsc8AuthorSpider(scrapy.Spider):
    name = 'gsc8author'
    allowed_domains = ['gsc8.net']
    start_urls = ['http://www.gsc8.net/listinfo-1-1.html']

    def parse(self, response):
        #获取下一页网址
        page_next = response.xpath('//*[@class="list_page"]//a[contains(text(),"下一页")]/@href')
        if page_next:
           yield scrapy.Request(response.urljoin( page_next.get() ), self.parse)
        #获取当前列表页所有的内容页网址
        for i in response.xpath("//*[@class=\"line_bottom\"]//strong/a/@href"):
            item_url = response.urljoin(i.get())
            yield scrapy.Request( item_url , self.get_item)
            
    def get_item(self, response):
        item = AuthorItem()
        #作者名字
        item['name'] = response.xpath("//h1/text()").extract_first()
        #作者照片
        item['image'] = response.urljoin( response.xpath("//*[@id=\"text\"]/p/a/img/@src").extract_first() )
        #朝代
        item['years'] = response.xpath("//*[@class=\"info_text\"][1]/a[1]/text()").extract_first()
        #内容
        item['intro'] = response.xpath("//*[@id=\"text\"]//text()").extract_first()
        #采集时间
        item['grab_time'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        #采集网址
        item['grab_url'] = response.request.url
        print("----------------------------")
        print(response.request.url)
        print("----------------------------")
        yield item


